﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Net;
using System.Collections;
using System.Text.RegularExpressions;
using System.Threading;


namespace g1
{
    class Crawler
    {
        private Hashtable urls = new Hashtable();
        private int count = 0;
        public string startUrl = "";
        //public string judge = "";
        public string message1 = "";
        public string message2 = "";
        public bool end = false;
        public void Crawl()
        {
            //Console.WriteLine("开始爬行了......");

            Console.WriteLine("爬行" + startUrl + "页面");
            string html = DownLoad(startUrl);//下载
                                             //(2)判断爬取的是否是html文本
                                             // Console.WriteLine(html);
                                             //Console.WriteLine(111);
            if (html.Substring(0, 15) == "<!DOCTYPE html>" || html.Substring(0, 15) == "<!doctype html>")
            {
                Parse(html);
                count++;
                Crawler2();
            }
            else
            {
                message2 = "爬取的不是HTML文本，结束";
                end = true;
                Console.WriteLine("爬取的不是HTML文本，结束。");
            }
        }

        public void Crawler2()
        {
            while (true)
            {
                string current = null;
                foreach (string url in urls.Keys)
                {//找到一个还未下载过的链接
                    if ((bool)urls[url]) continue;
                    current = url;
                }
                if (current == null) break;

                message1 += current + '\n';
                Console.WriteLine("爬行" + current + "页面！");
                DownLoad(current);
                urls[current] = true;
                count++;
            }
            end = true;
            Console.WriteLine("爬行结束");
        }
        public string DownLoad(string url)//下载网页
        {
            try
            {
                WebClient webclient = new WebClient();
                webclient.Encoding = Encoding.UTF8;
                string html = webclient.DownloadString(url);

                string filename = count.ToString() + ".txt";
                //(3)相对地址转为绝对地址爬取
                File.WriteAllText(filename, html, Encoding.UTF8);
                Console.WriteLine("爬取成功");
                return html;
            }
            catch (Exception ex)
            {
                message2 += ex.Message + '\n';
                Console.WriteLine("爬取失败" + ex.Message);
                return "";
            }
        }

        public void Parse(string html)//解析（只需解析一次）
        {
            string strRef = @"(href|HREF|src)[]*=[]*[""'][^""'#>]+[""']";
            MatchCollection matches = new Regex(strRef).Matches(html);
            foreach (Match match in matches)
            {
                strRef = match.Value.Substring(match.Value.IndexOf('=') + 1).Trim('"', '\"', '#', ' ', '>');
                //Console.WriteLine(strRef);
                if (strRef.Length == 0) continue;
                if (urls[strRef] == null)
                {
                    urls[strRef] = false;
                }
            }

        }
    }

}
